const Koa = require('koa2'),
	Router = require('koa-router'),
	cheerio = require('cheerio'),
	path = require('path'),
	superagent = require('superagent')
require('superagent-charset')(superagent)
let app = new Koa(),
	router = new Router();
let fs = require("fs");

const config = {
	host:'http://www.biquchi.com',
	startUrl:'/chapter/33859/313309.html',//开始章节的url
	nextBtnSelector:'.reader-main .section-opt a:last-child',//下一章按钮css class选择器
	titleSelector:'.reader-main h1.title',//章节标题选择器
	contentSelector:'#content p',//正文选择器
	filePath:'江湖夜雨十年灯.txt',//保存文件名，持续追加
	folder:'江湖夜雨十年灯',//文件夹
	charset:'utf-8',//编码 gbk
	checkNextUrl(url){
		// 判断是否结束
		if(url.indexOf('.html') > -1) return true;
		return false;
	},
};


// router.get('/', async (ctx, next) => {
// 	//开始爬取的章节
// 	getData(config.startUrl);
// 	ctx.body = "已开始爬取....."
// })

const createPath = (filename) => {
	return path.join(__dirname,filename);
}

function setFile(text,fileName='') {
	return new Promise((resolve, reject) => {
		// 把爬取的文章添加到文本后
		if(fileName){
			const filePath = createPath([config.folder,fileName].join('/')+'.txt');
			console.log('save path = ',filePath)
			fs.writeFile(filePath, text, function(err) {
				if (err) {
					reject(err)
				} else {
					resolve();
				}
			});
		}else{
			fs.appendFile(config.filePath, text, function(err) {
				if (err) {
					reject(err)
				} else {
					resolve();
				}
			});
		}
	})
}

//处理节点结构,获取节点内的文章数据
//本方法要根据爬取的网站的返回格式处理
function blChild(eleChild) {
	let strs = [];
	if (eleChild.length > 0) {
		for (let i = 0; i < eleChild.length; i++) {
			if (eleChild[i].type == "text" && eleChild[i].data) {
				// console.log('---',eleChild[i].data)
				strs.push(eleChild[i].data.trim());
			}
		}
		return strs.join('\n');
	}
}

//递归请求方法
let time = 0;
function getData(urlPath) {
	console.log('start down load url ' + urlPath)
	// if(time >= 1) return;
	// time += 1;
	const charset = config.charset;
	superagent.get(config.host + urlPath).charset(charset).buffer(true).then(
		(sres) => {
			if (sres) {
				// console.log(sres.text)
				$ = cheerio.load(sres.text, {
					decodeEntities: false
				}); //用cheerio解析页面数据
				// console.log($('head meta[http-equiv="Content-Type"]'))
				// console.log($('head meta[http-equiv="Content-Type"]')[0])
				const charset = ($('head meta[http-equiv="Content-Type"]')[0].attribs.content);
				if(charset.indexOf(charset) === -1){
					console.log('编码错误，',charset);
					return;
				}
				let yeStr = '',fileName = '';
				//获取当前章节目录名称
				$(config.titleSelector).each((ind, ele) => {
					if (ele.name == 'h1') {
						fileName = ele.children[0].data;
						console.log('fileName:',fileName)
						yeStr += fileName + '\r\n';
					}
				})
				// 正文
				$(config.contentSelector).each((ind, ele) => {
					let eleChild = ele.children;
					if(eleChild)
					{
						const p = blChild(eleChild);
						if(p) yeStr += p;
					}
				})
				// console.log(yeStr)
				// return
				//保存方法
				setFile(yeStr, fileName).then(() => {
					//递归调用,根据返回页面的"下一章"获取下一章文件的路径
					const nextUrl = $(config.nextBtnSelector)[0].attribs.href;
					if(config.checkNextUrl(nextUrl)){
						console.log('next url :',nextUrl)
						getData(nextUrl)
					}else{
						console.log('完成')
					}
				}).catch(() => {
					console.log('save failed')
					// getData(urlPath)
				});
			}
		}).catch((err) => {
			console.log('err')
			console.log(err)
			if (err) {
				console.log('err')
				getData(urlPath)
			}
		});
}
// app.use(router.routes()).use(router.allowedMethods());

// app.listen(3000, () => {
// 	console.log('[服务已开启,访问地址为：] http://127.0.0.1:3000/');
// });

getData(config.startUrl);
